{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c3eaa540-19d8-4e5e-9ab9-1845e7d35d6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/sagemaker-user/.config/sagemaker/config.yaml\n"
     ]
    }
   ],
   "source": [
    "import boto3, json, sagemaker, time, os\n",
    "from sagemaker import get_execution_role\n",
    "from pathlib import Path\n",
    "\n",
    "sess = boto3.Session()\n",
    "sm = sess.client(\"sagemaker\")\n",
    "sagemaker_session = sagemaker.Session(boto_session=sess)\n",
    "role = get_execution_role()\n",
    "client = boto3.client(\"sagemaker-runtime\")\n",
    "region = sess.region_name\n",
    "sts_client = sess.client('sts')\n",
    "account_id = sts_client.get_caller_identity()['Account']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f877e647-9149-4088-a638-be28a4527267",
   "metadata": {},
   "outputs": [],
   "source": [
    "# llama3-8b-instruct\n",
    "public_nim_image = \"public.ecr.aws/nvidia/nim:llama3-8b-instruct-1.0.0\"\n",
    "nim_model = \"nim-llama3-8b-instruct\"\n",
    "sm_model_name = \"nim-llama3-8b-instruct\"\n",
    "instance_type = \"ml.g5.4xlarge\"\n",
    "payload_model = \"meta/llama3-8b-instruct\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "97dff9c8-8834-4a55-819b-75c5c676a0f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AWS account ID: 039192856426\n",
      "Public NIM Image: public.ecr.aws/nvidia/nim:llama3-8b-instruct-1.0.0\n",
      "llama3-8b-instruct-1.0.0: Pulling from nvidia/nim\n",
      "cbe3537751ce: Pulling fs layer\n",
      "d67fcc6ef577: Pulling fs layer\n",
      "47ee674c5713: Pulling fs layer\n",
      "63daa0e64b30: Pulling fs layer\n",
      "d9d9aecefab5: Pulling fs layer\n",
      "d71f46a15657: Pulling fs layer\n",
      "054e2ffff644: Pulling fs layer\n",
      "7d3cd81654d5: Pulling fs layer\n",
      "dca613dca886: Pulling fs layer\n",
      "0fdcdcda3b2e: Pulling fs layer\n",
      "af7b4f7dc15a: Pulling fs layer\n",
      "6d101782f66c: Pulling fs layer\n",
      "e8427cb13897: Pulling fs layer\n",
      "de05b029a5a2: Pulling fs layer\n",
      "3d72a2698104: Pulling fs layer\n",
      "aeff973c2191: Pulling fs layer\n",
      "85d7d3ff0cca: Pulling fs layer\n",
      "5996430251dd: Pulling fs layer\n",
      "314dc83fdfc2: Pulling fs layer\n",
      "5cef8f59ae9a: Pulling fs layer\n",
      "927db4ce3e96: Pulling fs layer\n",
      "cbe4a04f4491: Pulling fs layer\n",
      "60f1a03c0955: Pulling fs layer\n",
      "67c1bb2b1aac: Pulling fs layer\n",
      "f16f7b821143: Pulling fs layer\n",
      "9be4fff0cd1a: Pulling fs layer\n",
      "d2bf4395eff6: Pulling fs layer\n",
      "606721f225c7: Pulling fs layer\n",
      "63daa0e64b30: Waiting\n",
      "d9d9aecefab5: Waiting\n",
      "d71f46a15657: Waiting\n",
      "054e2ffff644: Waiting\n",
      "7d3cd81654d5: Waiting\n",
      "dca613dca886: Waiting\n",
      "0fdcdcda3b2e: Waiting\n",
      "af7b4f7dc15a: Waiting\n",
      "6d101782f66c: Waiting\n",
      "e8427cb13897: Waiting\n",
      "cbe4a04f4491: Waiting\n",
      "60f1a03c0955: Waiting\n",
      "de05b029a5a2: Waiting\n",
      "3d72a2698104: Waiting\n",
      "aeff973c2191: Waiting\n",
      "67c1bb2b1aac: Waiting\n",
      "85d7d3ff0cca: Waiting\n",
      "5996430251dd: Waiting\n",
      "f16f7b821143: Waiting\n",
      "9be4fff0cd1a: Waiting\n",
      "d2bf4395eff6: Waiting\n",
      "606721f225c7: Waiting\n",
      "314dc83fdfc2: Waiting\n",
      "5cef8f59ae9a: Waiting\n",
      "927db4ce3e96: Waiting\n",
      "d67fcc6ef577: Verifying Checksum\n",
      "d67fcc6ef577: Download complete\n",
      "63daa0e64b30: Verifying Checksum\n",
      "63daa0e64b30: Download complete\n",
      "cbe3537751ce: Verifying Checksum\n",
      "cbe3537751ce: Download complete\n",
      "d9d9aecefab5: Verifying Checksum\n",
      "d9d9aecefab5: Download complete\n",
      "d71f46a15657: Verifying Checksum\n",
      "d71f46a15657: Download complete\n",
      "47ee674c5713: Verifying Checksum\n",
      "47ee674c5713: Download complete\n",
      "cbe3537751ce: Pull complete\n",
      "d67fcc6ef577: Pull complete\n",
      "dca613dca886: Verifying Checksum\n",
      "dca613dca886: Download complete\n",
      "0fdcdcda3b2e: Verifying Checksum\n",
      "0fdcdcda3b2e: Download complete\n",
      "47ee674c5713: Pull complete\n",
      "63daa0e64b30: Pull complete\n",
      "d9d9aecefab5: Pull complete\n",
      "054e2ffff644: Verifying Checksum\n",
      "054e2ffff644: Download complete\n",
      "6d101782f66c: Verifying Checksum\n",
      "6d101782f66c: Download complete\n",
      "e8427cb13897: Verifying Checksum\n",
      "e8427cb13897: Download complete\n",
      "de05b029a5a2: Verifying Checksum\n",
      "de05b029a5a2: Download complete\n",
      "3d72a2698104: Verifying Checksum\n",
      "3d72a2698104: Download complete\n",
      "aeff973c2191: Verifying Checksum\n",
      "aeff973c2191: Download complete\n",
      "85d7d3ff0cca: Verifying Checksum\n",
      "85d7d3ff0cca: Download complete\n",
      "d71f46a15657: Pull complete\n",
      "5996430251dd: Verifying Checksum\n",
      "5996430251dd: Download complete\n",
      "314dc83fdfc2: Verifying Checksum\n",
      "314dc83fdfc2: Download complete\n",
      "7d3cd81654d5: Verifying Checksum\n",
      "7d3cd81654d5: Download complete\n",
      "5cef8f59ae9a: Verifying Checksum\n",
      "5cef8f59ae9a: Download complete\n",
      "927db4ce3e96: Verifying Checksum\n",
      "927db4ce3e96: Download complete\n",
      "cbe4a04f4491: Verifying Checksum\n",
      "cbe4a04f4491: Download complete\n",
      "60f1a03c0955: Verifying Checksum\n",
      "60f1a03c0955: Download complete\n",
      "67c1bb2b1aac: Verifying Checksum\n",
      "67c1bb2b1aac: Download complete\n",
      "f16f7b821143: Verifying Checksum\n",
      "f16f7b821143: Download complete\n",
      "9be4fff0cd1a: Verifying Checksum\n",
      "9be4fff0cd1a: Download complete\n",
      "d2bf4395eff6: Verifying Checksum\n",
      "d2bf4395eff6: Download complete\n",
      "af7b4f7dc15a: Verifying Checksum\n",
      "af7b4f7dc15a: Download complete\n",
      "606721f225c7: Verifying Checksum\n",
      "606721f225c7: Download complete\n",
      "054e2ffff644: Pull complete\n",
      "7d3cd81654d5: Pull complete\n",
      "dca613dca886: Pull complete\n",
      "0fdcdcda3b2e: Pull complete\n",
      "af7b4f7dc15a: Pull complete\n",
      "6d101782f66c: Pull complete\n",
      "e8427cb13897: Pull complete\n",
      "de05b029a5a2: Pull complete\n",
      "3d72a2698104: Pull complete\n",
      "aeff973c2191: Pull complete\n",
      "85d7d3ff0cca: Pull complete\n",
      "5996430251dd: Pull complete\n",
      "314dc83fdfc2: Pull complete\n",
      "5cef8f59ae9a: Pull complete\n",
      "927db4ce3e96: Pull complete\n",
      "cbe4a04f4491: Pull complete\n",
      "60f1a03c0955: Pull complete\n",
      "67c1bb2b1aac: Pull complete\n",
      "f16f7b821143: Pull complete\n",
      "9be4fff0cd1a: Pull complete\n",
      "d2bf4395eff6: Pull complete\n",
      "606721f225c7: Pull complete\n",
      "Digest: sha256:ffd4f90eea16a22a17d9af15f5ae496202ae4c812241b6faeb1b554474486594\n",
      "Status: Downloaded newer image for public.ecr.aws/nvidia/nim:llama3-8b-instruct-1.0.0\n",
      "public.ecr.aws/nvidia/nim:llama3-8b-instruct-1.0.0\n",
      "Resolved account: 039192856426\n",
      "Resolved region: us-east-1\n",
      "Login Succeeded\n",
      "Using default tag: latest\n",
      "The push refers to repository [039192856426.dkr.ecr.us-east-1.amazonaws.com/nim-llama3-8b-instruct]\n",
      "7c18fcd44548: Preparing\n",
      "9dab84472c31: Preparing\n",
      "d6e3d1329879: Preparing\n",
      "c6efc285869e: Preparing\n",
      "2decb5c4804e: Preparing\n",
      "39111a8a9ded: Preparing\n",
      "ec561d7e6811: Preparing\n",
      "64698ac07231: Preparing\n",
      "086db532493b: Preparing\n",
      "96ffa74d03cf: Preparing\n",
      "029474febabe: Preparing\n",
      "591935392904: Preparing\n",
      "22ba5c2fc887: Preparing\n",
      "9034ce09b708: Preparing\n",
      "034e11c3e122: Preparing\n",
      "80e3915a53fa: Preparing\n",
      "2c9f8812e444: Preparing\n",
      "e9878d508160: Preparing\n",
      "e6a2e7dc7c95: Preparing\n",
      "57c2269cdf89: Preparing\n",
      "aa02d223d9df: Preparing\n",
      "17bbf87bba59: Preparing\n",
      "3970fd90d179: Preparing\n",
      "bae3163c64b8: Preparing\n",
      "f0fc8a1ca0cb: Preparing\n",
      "90efea7ecd8e: Preparing\n",
      "b6a0147bcf99: Preparing\n",
      "8ceb9643fb36: Preparing\n",
      "034e11c3e122: Waiting\n",
      "80e3915a53fa: Waiting\n",
      "2c9f8812e444: Waiting\n",
      "e9878d508160: Waiting\n",
      "e6a2e7dc7c95: Waiting\n",
      "57c2269cdf89: Waiting\n",
      "aa02d223d9df: Waiting\n",
      "17bbf87bba59: Waiting\n",
      "3970fd90d179: Waiting\n",
      "39111a8a9ded: Waiting\n",
      "ec561d7e6811: Waiting\n",
      "64698ac07231: Waiting\n",
      "086db532493b: Waiting\n",
      "96ffa74d03cf: Waiting\n",
      "029474febabe: Waiting\n",
      "591935392904: Waiting\n",
      "22ba5c2fc887: Waiting\n",
      "9034ce09b708: Waiting\n",
      "bae3163c64b8: Waiting\n",
      "f0fc8a1ca0cb: Waiting\n",
      "90efea7ecd8e: Waiting\n",
      "b6a0147bcf99: Waiting\n",
      "8ceb9643fb36: Waiting\n",
      "9dab84472c31: Layer already exists\n",
      "d6e3d1329879: Layer already exists\n",
      "2decb5c4804e: Layer already exists\n",
      "c6efc285869e: Layer already exists\n",
      "7c18fcd44548: Layer already exists\n",
      "39111a8a9ded: Layer already exists\n",
      "ec561d7e6811: Layer already exists\n",
      "64698ac07231: Layer already exists\n",
      "086db532493b: Layer already exists\n",
      "96ffa74d03cf: Layer already exists\n",
      "029474febabe: Layer already exists\n",
      "22ba5c2fc887: Layer already exists\n",
      "034e11c3e122: Layer already exists\n",
      "9034ce09b708: Layer already exists\n",
      "591935392904: Layer already exists\n",
      "80e3915a53fa: Layer already exists\n",
      "2c9f8812e444: Layer already exists\n",
      "57c2269cdf89: Layer already exists\n",
      "e9878d508160: Layer already exists\n",
      "e6a2e7dc7c95: Layer already exists\n",
      "aa02d223d9df: Layer already exists\n",
      "17bbf87bba59: Layer already exists\n",
      "bae3163c64b8: Layer already exists\n",
      "3970fd90d179: Layer already exists\n",
      "f0fc8a1ca0cb: Layer already exists\n",
      "90efea7ecd8e: Layer already exists\n",
      "b6a0147bcf99: Layer already exists\n",
      "8ceb9643fb36: Layer already exists\n",
      "latest: digest: sha256:4d383cbb31e710a6cd1bb1a443794f9e669fc5dd63df21c02e72166ac54eb415 size: 6184\n",
      "039192856426.dkr.ecr.us-east-1.amazonaws.com/nim-llama3-8b-instruct\n",
      "Errors: WARNING! Your password will be stored unencrypted in /home/sagemaker-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credential-stores\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import subprocess\n",
    "\n",
    "# Get AWS account ID\n",
    "result = subprocess.run(['aws', 'sts', 'get-caller-identity', '--query', 'Account', '--output', 'text'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)\n",
    "\n",
    "if result.returncode != 0:\n",
    "    print(f\"Error getting AWS account ID: {result.stderr}\")\n",
    "else:\n",
    "    account = result.stdout.strip()\n",
    "    print(f\"AWS account ID: {account}\")\n",
    "\n",
    "bash_script = f\"\"\"\n",
    "echo \"Public NIM Image: {public_nim_image}\"\n",
    "docker pull {public_nim_image}\n",
    "\n",
    "\n",
    "echo \"Resolved account: {account}\"\n",
    "echo \"Resolved region: {region}\"\n",
    "\n",
    "nim_image=\"{account}.dkr.ecr.{region}.amazonaws.com/{nim_model}\"\n",
    "\n",
    "# Ensure the repository name adheres to AWS constraints\n",
    "repository_name=$(echo \"{nim_model}\" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]._/-')\n",
    "\n",
    "# If the repository doesn't exist in ECR, create it.\n",
    "aws ecr describe-repositories --repository-names \"$repository_name\" > /dev/null 2>&1\n",
    "\n",
    "if [ $? -ne 0 ]\n",
    "then\n",
    "    aws ecr create-repository --repository-name \"$repository_name\" > /dev/null\n",
    "fi\n",
    "\n",
    "# Get the login command from ECR and execute it directly\n",
    "aws ecr get-login-password --region {region} | docker login --username AWS --password-stdin \"{account}.dkr.ecr.{region}.amazonaws.com\"\n",
    "\n",
    "docker tag {public_nim_image} $nim_image\n",
    "docker push $nim_image\n",
    "echo -n $nim_image\n",
    "\"\"\"\n",
    "nim_image=f\"{account}.dkr.ecr.{region}.amazonaws.com/{nim_model}\"\n",
    "# Run the bash script and capture real-time output\n",
    "process = subprocess.Popen(bash_script, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
    "\n",
    "while True:\n",
    "    output = process.stdout.readline()\n",
    "    if output == b'' and process.poll() is not None:\n",
    "        break\n",
    "    if output:\n",
    "        print(output.decode().strip())\n",
    "\n",
    "stderr = process.stderr.read().decode()\n",
    "if stderr:\n",
    "    print(\"Errors:\", stderr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "df4e809d-b4ce-470c-bdb2-c7106c22416a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "039192856426.dkr.ecr.us-east-1.amazonaws.com/nim-llama3-8b-instruct\n"
     ]
    }
   ],
   "source": [
    "print(nim_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fdc8532b-ada4-4a04-9db0-4ec0bb9f470c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# SET ME\n",
    "NGC_API_KEY = \"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\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "df59e833-4a5d-405c-b27f-c8809c7b5849",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Arn: arn:aws:sagemaker:us-east-1:039192856426:model/nim-llama3-8b-instruct\n"
     ]
    }
   ],
   "source": [
    "container = {\n",
    "    \"Image\": nim_image,\n",
    "    \"Environment\": {\"NGC_API_KEY\": NGC_API_KEY}\n",
    "}\n",
    "create_model_response = sm.create_model(\n",
    "    ModelName=sm_model_name, ExecutionRoleArn=role, PrimaryContainer=container\n",
    ")\n",
    "\n",
    "print(\"Model Arn: \" + create_model_response[\"ModelArn\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "85cfd697-326a-4195-9297-969b21e061d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Endpoint Config Arn: arn:aws:sagemaker:us-east-1:039192856426:endpoint-config/nim-llama3-8b-instruct\n"
     ]
    }
   ],
   "source": [
    "endpoint_config_name = sm_model_name\n",
    "\n",
    "create_endpoint_config_response = sm.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"InstanceType\": instance_type,\n",
    "            \"InitialVariantWeight\": 1,\n",
    "            \"InitialInstanceCount\": 1,\n",
    "            \"ModelName\": sm_model_name,\n",
    "            \"VariantName\": \"AllTraffic\",\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 1800,\n",
    "            \"InferenceAmiVersion\": \"al2-ami-sagemaker-inference-gpu-2\"\n",
    "        }\n",
    "    ],\n",
    ")\n",
    "\n",
    "print(\"Endpoint Config Arn: \" + create_endpoint_config_response[\"EndpointConfigArn\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "09820a3e-1684-42ca-a1dd-81a96afb8979",
   "metadata": {},
   "outputs": [
    {
     "ename": "ClientError",
     "evalue": "An error occurred (ValidationException) when calling the CreateEndpoint operation: Cannot create already existing endpoint \"arn:aws:sagemaker:us-east-1:039192856426:endpoint/nim-llama3-8b-instruct\".",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mClientError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[18], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m endpoint_name \u001b[38;5;241m=\u001b[39m sm_model_name\n\u001b[0;32m----> 3\u001b[0m create_endpoint_response \u001b[38;5;241m=\u001b[39m \u001b[43msm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_endpoint\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m    \u001b[49m\u001b[43mEndpointName\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mendpoint_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mEndpointConfigName\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mendpoint_config_name\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEndpoint Arn: \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m create_endpoint_response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEndpointArn\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n",
      "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/botocore/client.py:565\u001b[0m, in \u001b[0;36mClientCreator._create_api_method.<locals>._api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    561\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m    562\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpy_operation_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m() only accepts keyword arguments.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    563\u001b[0m     )\n\u001b[1;32m    564\u001b[0m \u001b[38;5;66;03m# The \"self\" in this scope is referring to the BaseClient.\u001b[39;00m\n\u001b[0;32m--> 565\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_api_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43moperation_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.11/site-packages/botocore/client.py:1017\u001b[0m, in \u001b[0;36mBaseClient._make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m   1013\u001b[0m     error_code \u001b[38;5;241m=\u001b[39m error_info\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mQueryErrorCode\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m error_info\u001b[38;5;241m.\u001b[39mget(\n\u001b[1;32m   1014\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCode\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1015\u001b[0m     )\n\u001b[1;32m   1016\u001b[0m     error_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mfrom_code(error_code)\n\u001b[0;32m-> 1017\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m error_class(parsed_response, operation_name)\n\u001b[1;32m   1018\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1019\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m parsed_response\n",
      "\u001b[0;31mClientError\u001b[0m: An error occurred (ValidationException) when calling the CreateEndpoint operation: Cannot create already existing endpoint \"arn:aws:sagemaker:us-east-1:039192856426:endpoint/nim-llama3-8b-instruct\"."
     ]
    }
   ],
   "source": [
    "endpoint_name = sm_model_name\n",
    "\n",
    "create_endpoint_response = sm.create_endpoint(\n",
    "    EndpointName=endpoint_name, EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "\n",
    "print(\"Endpoint Arn: \" + create_endpoint_response[\"EndpointArn\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "cce54375-f9e2-4537-b1d1-0e12e6ad223a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: InService\n",
      "Arn: arn:aws:sagemaker:us-east-1:039192856426:endpoint/nim-llama3-8b-instruct\n",
      "Status: InService\n"
     ]
    }
   ],
   "source": [
    "resp = sm.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp[\"EndpointStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp[\"EndpointStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
    "print(\"Status: \" + status)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "488d846d-b449-4ea7-a875-22b63068edd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"id\": \"cmpl-332d49e5a6c74a2bb60d7789165c8f1d\",\n",
      "  \"object\": \"chat.completion\",\n",
      "  \"created\": 1739161659,\n",
      "  \"model\": \"meta/llama3-8b-instruct\",\n",
      "  \"choices\": [\n",
      "    {\n",
      "      \"index\": 0,\n",
      "      \"message\": {\n",
      "        \"role\": \"assistant\",\n",
      "        \"content\": \"There once was a GPU so fine,\\nWhose parallel processing did shine.\\nIt crunched data with ease,\\nAnd worked with great peace,\\nSolving problems in a most divine!\"\n",
      "      },\n",
      "      \"logprobs\": null,\n",
      "      \"finish_reason\": \"stop\",\n",
      "      \"stop_reason\": 128009\n",
      "    }\n",
      "  ],\n",
      "  \"usage\": {\n",
      "    \"prompt_tokens\": 53,\n",
      "    \"total_tokens\": 90,\n",
      "    \"completion_tokens\": 37\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {\"role\": \"user\", \"content\": \"Hello! How are you?\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"Hi! I am quite well, how can I help you today?\"},\n",
    "    {\"role\": \"user\", \"content\": \"Write a short limerick about the wonders of GPU Computing.\"}\n",
    "]\n",
    "payload = {\n",
    "  \"model\": payload_model,\n",
    "  \"messages\": messages,\n",
    "  \"max_tokens\": 100\n",
    "}\n",
    "\n",
    "\n",
    "response = client.invoke_endpoint(\n",
    "    EndpointName=endpoint_name, ContentType=\"application/json\", Body=json.dumps(payload)\n",
    ")\n",
    "\n",
    "output = json.loads(response[\"Body\"].read().decode(\"utf8\"))\n",
    "print(json.dumps(output, indent=2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5e9fa0ae-7e53-4412-973f-f010f492ca3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = [\n",
    "    {\"role\": \"user\", \"content\": \"Hello! How are you?\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"Hi! I am quite well, how can I help you today?\"},\n",
    "    {\"role\": \"user\", \"content\": \"Write a short limerick about the wonders of GPU Computing.\"}\n",
    "]\n",
    "payload = {\n",
    "  \"model\": payload_model,\n",
    "  \"messages\": messages,\n",
    "  \"max_tokens\": 100,\n",
    "  \"stream\": True\n",
    "}\n",
    "\n",
    "\n",
    "response = client.invoke_endpoint_with_response_stream(\n",
    "    EndpointName=endpoint_name,\n",
    "    Body=json.dumps(payload),\n",
    "    ContentType=\"application/json\",\n",
    "    Accept=\"application/jsonlines\",\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a5a9c990-b0b6-4546-a1c9-c10432e96ebf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There once was a GPU so fine,\n",
      "Whose parallel processing did shine.\n",
      "It crunched data with ease,\n",
      "And worked with great peace,\n",
      "Solving problems in a most divine!"
     ]
    }
   ],
   "source": [
    "event_stream = response['Body']\n",
    "accumulated_data = \"\"\n",
    "start_marker = 'data:'\n",
    "end_marker = '\"finish_reason\":null}]}'\n",
    "\n",
    "for event in event_stream:\n",
    "    try:\n",
    "        payload = event.get('PayloadPart', {}).get('Bytes', b'')\n",
    "        if payload:\n",
    "            data_str = payload.decode('utf-8')\n",
    "\n",
    "            accumulated_data += data_str\n",
    "\n",
    "            # Process accumulated data when a complete response is detected\n",
    "            while start_marker in accumulated_data and end_marker in accumulated_data:\n",
    "                start_idx = accumulated_data.find(start_marker)\n",
    "                end_idx = accumulated_data.find(end_marker) + len(end_marker)\n",
    "                full_response = accumulated_data[start_idx + len(start_marker):end_idx]\n",
    "                accumulated_data = accumulated_data[end_idx:]\n",
    "\n",
    "                try:\n",
    "                    data = json.loads(full_response)\n",
    "                    content = data.get('choices', [{}])[0].get('delta', {}).get('content', \"\")\n",
    "                    if content:\n",
    "                        print(content, end='', flush=True)\n",
    "                except json.JSONDecodeError:\n",
    "                    continue\n",
    "    except Exception as e:\n",
    "        print(f\"\\nError processing event: {e}\", flush=True)\n",
    "        continue\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "84afd6d7-c423-4be4-9b6f-ecca39c916fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ResponseMetadata': {'RequestId': 'e983d681-8427-43aa-8f97-6470c850bc6d',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': 'e983d681-8427-43aa-8f97-6470c850bc6d',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'date': 'Tue, 14 Jan 2025 07:33:51 GMT',\n",
       "   'content-length': '0'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm.delete_model(ModelName=sm_model_name)\n",
    "sm.delete_endpoint_config(EndpointConfigName=endpoint_config_name)\n",
    "sm.delete_endpoint(EndpointName=endpoint_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05a65f41-848f-4e80-92ac-454c79d798bd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
